In [1]:
from tensorflow.keras import backend as K
import gc
K.clear_session()
gc.collect()

WARNING:tensorflow:From C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\backend\common\global_state.py:82: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.
Out[1]:
0

Multiclass classification¶

Step1: Data Preparation¶

In [2]:
import numpy as np
import matplotlib.pyplot as plt
import h5py
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.utils import to_categorical
from sklearn.utils import class_weight
from tensorflow.keras.metrics import Precision, Recall, AUC
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout
In [3]:
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
In [4]:
# Load the HDF5 file
bird_sound = h5py.File("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\\Homework3\\bird_spectrograms.hdf5", 'r')
In [5]:
species= list(bird_sound.keys())

# Load all spectrograms and labels
X = []
y = []

for label in species:
    data =bird_sound[label][:]  
    data = np.transpose(data, (2, 0, 1))  
    X.append(data)
    y += [label] * data.shape[0]


X = np.concatenate(X, axis=0)
print(X.shape) 

(1981, 128, 517)
In [6]:
# Plot one spectrogram sample from each species
unique_species = list(np.unique(y))
sample_specs = {}

for label in unique_species:
    label_index = y.index(label) if isinstance(y, list) else np.where(np.array(y) == label)[0][0]
    sample = X[label_index, :, :]

    sample_specs[label] = sample

plt.figure(figsize=(15, 3 * len(sample_specs)))

for i, (label, spec) in enumerate(sample_specs.items()):
    plt.subplot(len(sample_specs), 1, i + 1)
    plt.imshow(spec, aspect='auto', origin='lower', cmap='inferno')
    plt.title(f"Species: {label}")
    plt.colorbar()
    plt.axis('off')

plt.tight_layout()
plt.show()
No description has been provided for this image

Step2: Train-Test Split¶

In [7]:
X = X.reshape((1981, 128, 517, 1))
y = np.array(y)
label_encoder = LabelEncoder()
y_encoded = label_encoder.fit_transform(y)
y_categorical = to_categorical(y_encoded)


# Train-test split
X_train, X_test, y_train, y_test = train_test_split(
    X, y_categorical, test_size=0.2, random_state=156, stratify=y_encoded
)
print(f"X_train shape: {X_train.shape}, y_train shape: {y_train.shape}")
print(f"X_test shape: {X_test.shape}, y_test shape: {y_test.shape}")
print(f"Number of classes: {len(label_encoder.classes_)}")

X_train shape: (1584, 128, 517, 1), y_train shape: (1584, 12)
X_test shape: (397, 128, 517, 1), y_test shape: (397, 12)
Number of classes: 12

Step3: Model Training¶

Model1¶
In [8]:
# Definigng model1
np.random.seed(123)
tf.random.set_seed(123)

model1_multi = Sequential()
model1_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1_multi.add(Flatten())
model1_multi.add(Dense(128, activation='relu'))
model1_multi.add(Dropout(0.5))

model1_multi.add(Dense(12, activation='softmax'))  


model1_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [9]:
import time
start_time = time.time()

history1 = model1_multi.fit(
    X_train, y_train,
    validation_data=(X_test, y_test),
    epochs=20,
    batch_size=256
)

end_time = time.time()
time1 = end_time - start_time
print(f"\nTraining time: {time1:.2f} seconds")

Epoch 1/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.1411 - auc: 0.5307 - loss: 545.9827 - precision: 0.1426 - recall: 0.1411 - val_accuracy: 0.0453 - val_auc: 0.4780 - val_loss: 59.9108 - val_precision: 0.0455 - val_recall: 0.0453
Epoch 2/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.1031 - auc: 0.5173 - loss: 44.2694 - precision: 0.1022 - recall: 0.0866 - val_accuracy: 0.3073 - val_auc: 0.5002 - val_loss: 2.4888 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 3/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.2602 - auc: 0.5037 - loss: 3.2944 - precision: 0.0763 - recall: 0.0186 - val_accuracy: 0.0982 - val_auc: 0.4269 - val_loss: 2.9123 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 4/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.1834 - auc: 0.5233 - loss: 2.7006 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.3476 - val_auc: 0.6964 - val_loss: 2.4225 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 5/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.3829 - auc: 0.7179 - loss: 2.3431 - precision: 0.3555 - recall: 0.0182 - val_accuracy: 0.3602 - val_auc: 0.7486 - val_loss: 2.2619 - val_precision: 1.0000 - val_recall: 0.0025
Epoch 6/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.4012 - auc: 0.7967 - loss: 2.0732 - precision: 0.8099 - recall: 0.0506 - val_accuracy: 0.4030 - val_auc: 0.7713 - val_loss: 2.0368 - val_precision: 0.5932 - val_recall: 0.0882
Epoch 7/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.4880 - auc: 0.8424 - loss: 1.7971 - precision: 0.6669 - recall: 0.1303 - val_accuracy: 0.4081 - val_auc: 0.7851 - val_loss: 2.0887 - val_precision: 0.5025 - val_recall: 0.2569
Epoch 8/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 4s/step - accuracy: 0.5516 - auc: 0.8821 - loss: 1.5779 - precision: 0.7027 - recall: 0.2674 - val_accuracy: 0.4131 - val_auc: 0.7977 - val_loss: 1.9996 - val_precision: 0.5183 - val_recall: 0.2141
Epoch 9/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 33s 5s/step - accuracy: 0.6463 - auc: 0.9291 - loss: 1.2091 - precision: 0.7768 - recall: 0.4323 - val_accuracy: 0.3703 - val_auc: 0.7754 - val_loss: 2.2326 - val_precision: 0.4770 - val_recall: 0.2091
Epoch 10/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.7044 - auc: 0.9524 - loss: 1.0085 - precision: 0.8508 - recall: 0.4959 - val_accuracy: 0.4257 - val_auc: 0.8066 - val_loss: 2.0428 - val_precision: 0.5694 - val_recall: 0.2997
Epoch 11/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 33s 5s/step - accuracy: 0.7876 - auc: 0.9773 - loss: 0.7180 - precision: 0.9011 - recall: 0.6416 - val_accuracy: 0.4408 - val_auc: 0.8011 - val_loss: 2.0789 - val_precision: 0.5631 - val_recall: 0.2922
Epoch 12/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.8431 - auc: 0.9899 - loss: 0.5213 - precision: 0.9326 - recall: 0.7390 - val_accuracy: 0.4332 - val_auc: 0.8012 - val_loss: 2.1565 - val_precision: 0.5402 - val_recall: 0.3048
Epoch 13/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.9085 - auc: 0.9956 - loss: 0.3503 - precision: 0.9638 - recall: 0.8365 - val_accuracy: 0.4559 - val_auc: 0.7993 - val_loss: 2.3629 - val_precision: 0.5349 - val_recall: 0.3476
Epoch 14/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9285 - auc: 0.9976 - loss: 0.2582 - precision: 0.9752 - recall: 0.8818 - val_accuracy: 0.4307 - val_auc: 0.8021 - val_loss: 2.4970 - val_precision: 0.5068 - val_recall: 0.3778
Epoch 15/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.9570 - auc: 0.9992 - loss: 0.1659 - precision: 0.9755 - recall: 0.9340 - val_accuracy: 0.4408 - val_auc: 0.8051 - val_loss: 2.5254 - val_precision: 0.4765 - val_recall: 0.3577
Epoch 16/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9673 - auc: 0.9994 - loss: 0.1530 - precision: 0.9851 - recall: 0.9394 - val_accuracy: 0.4257 - val_auc: 0.7952 - val_loss: 2.8406 - val_precision: 0.4763 - val_recall: 0.3804
Epoch 17/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9562 - auc: 0.9993 - loss: 0.1310 - precision: 0.9726 - recall: 0.9422 - val_accuracy: 0.4710 - val_auc: 0.8041 - val_loss: 2.5447 - val_precision: 0.5288 - val_recall: 0.3929
Epoch 18/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9813 - auc: 0.9996 - loss: 0.0987 - precision: 0.9884 - recall: 0.9640 - val_accuracy: 0.4332 - val_auc: 0.8000 - val_loss: 2.6519 - val_precision: 0.4901 - val_recall: 0.3753
Epoch 19/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9797 - auc: 0.9997 - loss: 0.0756 - precision: 0.9891 - recall: 0.9743 - val_accuracy: 0.4257 - val_auc: 0.7952 - val_loss: 2.9259 - val_precision: 0.4985 - val_recall: 0.4106
Epoch 20/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9854 - auc: 0.9999 - loss: 0.0571 - precision: 0.9896 - recall: 0.9803 - val_accuracy: 0.4332 - val_auc: 0.7920 - val_loss: 2.7538 - val_precision: 0.4903 - val_recall: 0.3829

Training time: 641.04 seconds
In [10]:
print("\nValidation Accuracy Model1:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history1.history[m][-1]:.4f}")

Validation Accuracy Model1:
val_accuracy: 0.4332
val_precision: 0.4903
val_recall: 0.3829
val_auc: 0.7920
  • Model 1 has validation accuracy of (43.32%) and a strong AUC of 0.7844. These results indicate effective overall classification across multiple bird species.
In [11]:
# plotting the results
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history1.history['accuracy'], label='Train Accuracy')
plt.plot(history1.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history1.history['loss'], label='Train Loss')
plt.plot(history1.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
  • The graphs show that the model's training accuracy keeps improving, but the validation accuracy stays around 45%, which means the model might be overfitting.
  • The loss graph also shows training loss goes down , while validation loss remains mostly flat indicatingthat the model is learning the training data well but not generalizing to new data.
In [12]:
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

y_pred = model1_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm1 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm1, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 117ms/step
No description has been provided for this image
  • This confusion matrix shows that the model performs best on houspa (House Sparrow), predicting it correctly 106 times. However, it struggles to distinguish between several other bird species like bewwre, rewbla, and sonspa, often confusing them with each other.
Model2¶
In [13]:
# Defining model2
np.random.seed(42)
tf.random.set_seed(42)


model2_multi = Sequential()
model2_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2_multi.add(Flatten())
model2_multi.add(Dense(64, activation='relu'))
model2_multi.add(Dropout(0.3))

model2_multi.add(Dense(12, activation='softmax'))  

model2_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [14]:
# Tracking the model training time
import time
start_time = time.time()
history2 = model2_multi.fit(
    X_train, y_train,
    validation_data=(X_test, y_test),
    epochs=15,
    batch_size=128
)

end_time = time.time()
time2 = end_time - start_time
print(f"\nTraining time: {time2:.2f} seconds")

Epoch 1/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 11s 694ms/step - accuracy: 0.1675 - auc: 0.5497 - loss: 225.6911 - precision: 0.1647 - recall: 0.1556 - val_accuracy: 0.3174 - val_auc: 0.6720 - val_loss: 2.3636 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 654ms/step - accuracy: 0.1788 - auc: 0.6281 - loss: 2.4847 - precision: 0.0977 - recall: 0.0025 - val_accuracy: 0.2771 - val_auc: 0.6823 - val_loss: 2.3905 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 3/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 670ms/step - accuracy: 0.2556 - auc: 0.6828 - loss: 2.4334 - precision: 0.3094 - recall: 0.0339 - val_accuracy: 0.3149 - val_auc: 0.7282 - val_loss: 2.2707 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 4/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 681ms/step - accuracy: 0.3738 - auc: 0.7624 - loss: 2.1663 - precision: 0.5728 - recall: 0.0493 - val_accuracy: 0.3778 - val_auc: 0.7536 - val_loss: 2.1355 - val_precision: 0.3929 - val_recall: 0.0554
Epoch 5/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 676ms/step - accuracy: 0.5174 - auc: 0.8676 - loss: 1.7304 - precision: 0.6833 - recall: 0.1256 - val_accuracy: 0.4081 - val_auc: 0.7968 - val_loss: 1.9734 - val_precision: 0.5104 - val_recall: 0.1234
Epoch 6/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 668ms/step - accuracy: 0.6408 - auc: 0.9278 - loss: 1.2562 - precision: 0.8154 - recall: 0.3680 - val_accuracy: 0.4131 - val_auc: 0.7979 - val_loss: 2.0009 - val_precision: 0.5152 - val_recall: 0.1285
Epoch 7/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 661ms/step - accuracy: 0.7841 - auc: 0.9761 - loss: 0.7922 - precision: 0.9134 - recall: 0.5615 - val_accuracy: 0.4307 - val_auc: 0.7871 - val_loss: 2.3146 - val_precision: 0.4957 - val_recall: 0.2897
Epoch 8/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 653ms/step - accuracy: 0.8689 - auc: 0.9894 - loss: 0.4926 - precision: 0.9371 - recall: 0.7634 - val_accuracy: 0.4358 - val_auc: 0.8013 - val_loss: 2.2728 - val_precision: 0.4791 - val_recall: 0.3174
Epoch 9/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 661ms/step - accuracy: 0.9118 - auc: 0.9940 - loss: 0.3379 - precision: 0.9584 - recall: 0.8525 - val_accuracy: 0.4282 - val_auc: 0.7932 - val_loss: 2.2156 - val_precision: 0.5138 - val_recall: 0.2821
Epoch 10/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 671ms/step - accuracy: 0.9505 - auc: 0.9960 - loss: 0.2284 - precision: 0.9681 - recall: 0.9132 - val_accuracy: 0.4005 - val_auc: 0.7658 - val_loss: 2.6874 - val_precision: 0.4648 - val_recall: 0.2997
Epoch 11/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 693ms/step - accuracy: 0.9315 - auc: 0.9961 - loss: 0.3092 - precision: 0.9578 - recall: 0.8611 - val_accuracy: 0.4207 - val_auc: 0.7729 - val_loss: 2.5517 - val_precision: 0.4835 - val_recall: 0.3325
Epoch 12/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 686ms/step - accuracy: 0.9635 - auc: 0.9990 - loss: 0.1784 - precision: 0.9795 - recall: 0.9316 - val_accuracy: 0.4106 - val_auc: 0.7864 - val_loss: 2.6806 - val_precision: 0.4804 - val_recall: 0.3401
Epoch 13/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 687ms/step - accuracy: 0.9716 - auc: 0.9991 - loss: 0.1314 - precision: 0.9805 - recall: 0.9521 - val_accuracy: 0.3980 - val_auc: 0.7801 - val_loss: 2.9548 - val_precision: 0.4441 - val_recall: 0.3300
Epoch 14/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 671ms/step - accuracy: 0.9817 - auc: 0.9995 - loss: 0.0800 - precision: 0.9859 - recall: 0.9755 - val_accuracy: 0.4257 - val_auc: 0.7916 - val_loss: 2.8339 - val_precision: 0.5017 - val_recall: 0.3728
Epoch 15/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 668ms/step - accuracy: 0.9873 - auc: 0.9997 - loss: 0.0545 - precision: 0.9915 - recall: 0.9847 - val_accuracy: 0.4635 - val_auc: 0.7810 - val_loss: 3.0003 - val_precision: 0.5081 - val_recall: 0.3929

Training time: 134.07 seconds
In [15]:
print("\nValidation Accuracy Model2:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history2.history[m][-1]:.4f}")

Validation Accuracy Model2:
val_accuracy: 0.4635
val_precision: 0.5081
val_recall: 0.3929
val_auc: 0.7810
  • Model 2 has validation accuracy of (46.35%) and a strong AUC of 0.7810. These results shows that it has 3% moere accuracy than model1.
In [16]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history2.history['accuracy'], label='Train Accuracy')
plt.plot(history2.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history2.history['loss'], label='Train Loss')
plt.plot(history2.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [17]:
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

y_pred = model2_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm2 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm2, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 1s 48ms/step
No description has been provided for this image
  • The model predicts House Sparrow and song sparrow very accurately, with minimal confusion.

Model1-A¶

In [18]:
np.random.seed(123)
tf.random.set_seed(123)

model1a_multi = Sequential()
model1a_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1a_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1a_multi.add(Flatten())
model1a_multi.add(Dense(128, activation='relu'))
model1a_multi.add(Dropout(0.4))

model1a_multi.add(Dense(12, activation='softmax'))  

model1a_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [19]:
start_time = time.time()
history3 = model1a_multi.fit(
    X_train, y_train,
    validation_data=(X_test, y_test),
    epochs=20,
    batch_size=128
)

end_time = time.time()
time3 = end_time - start_time
print(f"\nTraining time: {time3:.2f} seconds")

Epoch 1/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 2s/step - accuracy: 0.1474 - auc: 0.5388 - loss: 490.2636 - precision: 0.1479 - recall: 0.1436 - val_accuracy: 0.0227 - val_auc: 0.5114 - val_loss: 2.4803 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.1077 - auc: 0.5389 - loss: 4.6898 - precision: 0.1610 - recall: 0.0580 - val_accuracy: 0.1511 - val_auc: 0.5459 - val_loss: 2.4739 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 3/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.1850 - auc: 0.5440 - loss: 2.4723 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.3174 - val_auc: 0.5255 - val_loss: 2.4738 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 4/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.3251 - auc: 0.6218 - loss: 2.4475 - precision: 0.2931 - recall: 0.0156 - val_accuracy: 0.3325 - val_auc: 0.6755 - val_loss: 2.4626 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 5/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.3568 - auc: 0.7039 - loss: 2.4295 - precision: 0.2080 - recall: 0.0055 - val_accuracy: 0.3073 - val_auc: 0.7320 - val_loss: 2.3537 - val_precision: 0.8000 - val_recall: 0.0101
Epoch 6/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.3726 - auc: 0.7713 - loss: 2.2719 - precision: 0.9558 - recall: 0.0228 - val_accuracy: 0.3627 - val_auc: 0.7614 - val_loss: 2.1646 - val_precision: 0.2973 - val_recall: 0.0277
Epoch 7/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.4831 - auc: 0.8300 - loss: 1.8891 - precision: 0.6481 - recall: 0.1299 - val_accuracy: 0.4232 - val_auc: 0.7944 - val_loss: 2.0609 - val_precision: 0.5044 - val_recall: 0.1436
Epoch 8/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.5993 - auc: 0.9144 - loss: 1.3373 - precision: 0.7948 - recall: 0.3693 - val_accuracy: 0.3904 - val_auc: 0.7869 - val_loss: 2.1221 - val_precision: 0.4809 - val_recall: 0.1587
Epoch 9/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.7471 - auc: 0.9612 - loss: 0.9261 - precision: 0.8851 - recall: 0.5404 - val_accuracy: 0.3678 - val_auc: 0.7654 - val_loss: 2.3917 - val_precision: 0.4556 - val_recall: 0.2065
Epoch 10/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.8065 - auc: 0.9766 - loss: 0.7406 - precision: 0.9126 - recall: 0.6287 - val_accuracy: 0.4181 - val_auc: 0.7855 - val_loss: 2.6065 - val_precision: 0.4743 - val_recall: 0.3023
Epoch 11/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.8809 - auc: 0.9901 - loss: 0.4354 - precision: 0.9246 - recall: 0.7986 - val_accuracy: 0.3955 - val_auc: 0.7822 - val_loss: 2.6364 - val_precision: 0.4563 - val_recall: 0.2897
Epoch 12/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.9286 - auc: 0.9970 - loss: 0.2632 - precision: 0.9641 - recall: 0.8865 - val_accuracy: 0.3955 - val_auc: 0.7760 - val_loss: 2.9961 - val_precision: 0.4621 - val_recall: 0.3375
Epoch 13/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 31s 2s/step - accuracy: 0.9571 - auc: 0.9989 - loss: 0.1614 - precision: 0.9749 - recall: 0.9297 - val_accuracy: 0.3955 - val_auc: 0.7766 - val_loss: 3.1419 - val_precision: 0.4400 - val_recall: 0.3325
Epoch 14/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.9781 - auc: 0.9994 - loss: 0.1087 - precision: 0.9875 - recall: 0.9576 - val_accuracy: 0.3980 - val_auc: 0.7760 - val_loss: 3.2108 - val_precision: 0.4295 - val_recall: 0.3375
Epoch 15/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9758 - auc: 0.9994 - loss: 0.0937 - precision: 0.9864 - recall: 0.9628 - val_accuracy: 0.4106 - val_auc: 0.7697 - val_loss: 3.4532 - val_precision: 0.4303 - val_recall: 0.3501
Epoch 16/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 31s 2s/step - accuracy: 0.9811 - auc: 0.9996 - loss: 0.0762 - precision: 0.9842 - recall: 0.9718 - val_accuracy: 0.3854 - val_auc: 0.7689 - val_loss: 3.5760 - val_precision: 0.4149 - val_recall: 0.3375
Epoch 17/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 3s/step - accuracy: 0.9813 - auc: 0.9991 - loss: 0.0869 - precision: 0.9856 - recall: 0.9717 - val_accuracy: 0.3929 - val_auc: 0.7609 - val_loss: 3.6187 - val_precision: 0.4182 - val_recall: 0.3350
Epoch 18/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9829 - auc: 0.9993 - loss: 0.0803 - precision: 0.9859 - recall: 0.9692 - val_accuracy: 0.3829 - val_auc: 0.7631 - val_loss: 3.6721 - val_precision: 0.4322 - val_recall: 0.3451
Epoch 19/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9850 - auc: 0.9996 - loss: 0.0590 - precision: 0.9900 - recall: 0.9770 - val_accuracy: 0.3879 - val_auc: 0.7549 - val_loss: 3.8591 - val_precision: 0.4218 - val_recall: 0.3602
Epoch 20/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.9867 - auc: 0.9999 - loss: 0.0456 - precision: 0.9921 - recall: 0.9839 - val_accuracy: 0.4055 - val_auc: 0.7673 - val_loss: 3.8660 - val_precision: 0.4332 - val_recall: 0.3678

Training time: 659.30 seconds
In [20]:
print("\nValidation Accuracy Model3:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history3.history[m][-1]:.4f}")

Validation Accuracy Model3:
val_accuracy: 0.4055
val_precision: 0.4332
val_recall: 0.3678
val_auc: 0.7673
In [21]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history3.history['accuracy'], label='Train Accuracy')
plt.plot(history3.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history3.history['loss'], label='Train Loss')
plt.plot(history3.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [22]:
y_pred = model1a_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm3 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm3, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 125ms/step
No description has been provided for this image
Model1-B¶
In [23]:
np.random.seed(160)
tf.random.set_seed(160)

model1b_multi = Sequential()
model1b_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1b_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1b_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1b_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1b_multi.add(Flatten())
model1b_multi.add(Dense(128, activation='relu'))
model1b_multi.add(Dropout(0.3))

model1b_multi.add(Dense(12, activation='softmax'))  

model1b_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [24]:
start_time = time.time()

history4= model1b_multi.fit(
    X_train, y_train,
    epochs=15,
    batch_size=64,
    validation_data=(X_test, y_test)
)

end_time = time.time()
time4 = end_time - start_time

print(f"Training time: {time4:.2f} seconds ({time4 / 60:.2f} minutes)")

Epoch 1/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 37s 1s/step - accuracy: 0.1688 - auc: 0.5577 - loss: 277.2318 - precision: 0.1606 - recall: 0.1203 - val_accuracy: 0.3652 - val_auc: 0.7518 - val_loss: 2.2955 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 34s 1s/step - accuracy: 0.3851 - auc: 0.7703 - loss: 2.1832 - precision: 0.5867 - recall: 0.0304 - val_accuracy: 0.3955 - val_auc: 0.7954 - val_loss: 2.0291 - val_precision: 0.5455 - val_recall: 0.0453
Epoch 3/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.5282 - auc: 0.8698 - loss: 1.6901 - precision: 0.7639 - recall: 0.1532 - val_accuracy: 0.4257 - val_auc: 0.8044 - val_loss: 1.9602 - val_precision: 0.5440 - val_recall: 0.1713
Epoch 4/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.6821 - auc: 0.9448 - loss: 1.1306 - precision: 0.8553 - recall: 0.4145 - val_accuracy: 0.3678 - val_auc: 0.8055 - val_loss: 2.2662 - val_precision: 0.4735 - val_recall: 0.2695
Epoch 5/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.7801 - auc: 0.9670 - loss: 0.8575 - precision: 0.8845 - recall: 0.5553 - val_accuracy: 0.4433 - val_auc: 0.8014 - val_loss: 2.3811 - val_precision: 0.5075 - val_recall: 0.3401
Epoch 6/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.8629 - auc: 0.9911 - loss: 0.4389 - precision: 0.9223 - recall: 0.7953 - val_accuracy: 0.4307 - val_auc: 0.7871 - val_loss: 2.8283 - val_precision: 0.4704 - val_recall: 0.3602
Epoch 7/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 35s 1s/step - accuracy: 0.9368 - auc: 0.9979 - loss: 0.2201 - precision: 0.9600 - recall: 0.8997 - val_accuracy: 0.4358 - val_auc: 0.7741 - val_loss: 3.3975 - val_precision: 0.4688 - val_recall: 0.3980
Epoch 8/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 35s 1s/step - accuracy: 0.9330 - auc: 0.9913 - loss: 0.3123 - precision: 0.9470 - recall: 0.9139 - val_accuracy: 0.4030 - val_auc: 0.7788 - val_loss: 2.8058 - val_precision: 0.4586 - val_recall: 0.3350
Epoch 9/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9369 - auc: 0.9922 - loss: 0.2830 - precision: 0.9527 - recall: 0.8920 - val_accuracy: 0.4559 - val_auc: 0.7755 - val_loss: 3.2140 - val_precision: 0.5061 - val_recall: 0.4207
Epoch 10/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9678 - auc: 0.9993 - loss: 0.1325 - precision: 0.9752 - recall: 0.9502 - val_accuracy: 0.4282 - val_auc: 0.7860 - val_loss: 3.2643 - val_precision: 0.4679 - val_recall: 0.3854
Epoch 11/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9704 - auc: 0.9995 - loss: 0.1152 - precision: 0.9789 - recall: 0.9520 - val_accuracy: 0.4660 - val_auc: 0.7932 - val_loss: 3.1306 - val_precision: 0.4882 - val_recall: 0.4156
Epoch 12/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9834 - auc: 0.9991 - loss: 0.0644 - precision: 0.9870 - recall: 0.9752 - val_accuracy: 0.4232 - val_auc: 0.7987 - val_loss: 3.3640 - val_precision: 0.4599 - val_recall: 0.3904
Epoch 13/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 34s 1s/step - accuracy: 0.9837 - auc: 0.9998 - loss: 0.0568 - precision: 0.9879 - recall: 0.9802 - val_accuracy: 0.4358 - val_auc: 0.7812 - val_loss: 3.7600 - val_precision: 0.4718 - val_recall: 0.4207
Epoch 14/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9933 - auc: 1.0000 - loss: 0.0276 - precision: 0.9953 - recall: 0.9916 - val_accuracy: 0.4332 - val_auc: 0.7918 - val_loss: 3.4574 - val_precision: 0.4687 - val_recall: 0.3955
Epoch 15/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9900 - auc: 0.9999 - loss: 0.0329 - precision: 0.9932 - recall: 0.9883 - val_accuracy: 0.4484 - val_auc: 0.7943 - val_loss: 3.4947 - val_precision: 0.4926 - val_recall: 0.4207
Training time: 506.91 seconds (8.45 minutes)
In [25]:
print("\nValidation Accuracy Model4:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history4.history[m][-1]:.4f}")

Validation Accuracy Model4:
val_accuracy: 0.4484
val_precision: 0.4926
val_recall: 0.4207
val_auc: 0.7943
In [26]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history4.history['accuracy'], label='Train Accuracy')
plt.plot(history4.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history4.history['loss'], label='Train Loss')
plt.plot(history4.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [27]:
y_pred = model1b_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm4 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm4, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 121ms/step
No description has been provided for this image

Model2-A¶

In [28]:
np.random.seed(160)
tf.random.set_seed(160)


model2a_multi = Sequential()
model2a_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2a_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2a_multi.add(Flatten())
model2a_multi.add(Dense(64, activation='relu'))
model2a_multi.add(Dropout(0.4))

model2a_multi.add(Dense(12, activation='softmax'))  

model2a_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

model2a_multi.summary()

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Model: "sequential_4"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ conv2d_8 (Conv2D)               │ (None, 126, 515, 16)   │           160 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ max_pooling2d_8 (MaxPooling2D)  │ (None, 63, 257, 16)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv2d_9 (Conv2D)               │ (None, 61, 255, 32)    │         4,640 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ max_pooling2d_9 (MaxPooling2D)  │ (None, 30, 127, 32)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ flatten_4 (Flatten)             │ (None, 121920)         │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_8 (Dense)                 │ (None, 64)             │     7,802,944 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dropout_4 (Dropout)             │ (None, 64)             │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_9 (Dense)                 │ (None, 12)             │           780 │
└─────────────────────────────────┴────────────────────────┴───────────────┘

 Total params: 7,808,524 (29.79 MB)

 Trainable params: 7,808,524 (29.79 MB)

 Non-trainable params: 0 (0.00 B)
In [29]:
start_time = time.time()
history6= model2a_multi.fit(
    X_train, y_train,
    epochs=15,
    batch_size=32,
    validation_data=(X_test, y_test)
)

end_time = time.time()
time6 = end_time - start_time

print(f"Training time: {time6:.2f} seconds ({time6 / 60:.2f} minutes)")

Epoch 1/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 13s 218ms/step - accuracy: 0.1576 - auc: 0.5262 - loss: 89.9453 - precision: 0.1160 - recall: 0.0574 - val_accuracy: 0.2796 - val_auc: 0.6792 - val_loss: 2.3146 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 211ms/step - accuracy: 0.2761 - auc: 0.6786 - loss: 2.3032 - precision: 0.6057 - recall: 0.0140 - val_accuracy: 0.3325 - val_auc: 0.7132 - val_loss: 2.1947 - val_precision: 0.5946 - val_recall: 0.0554
Epoch 3/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 210ms/step - accuracy: 0.3382 - auc: 0.7604 - loss: 2.0945 - precision: 0.7583 - recall: 0.0832 - val_accuracy: 0.3804 - val_auc: 0.7514 - val_loss: 2.1184 - val_precision: 0.7500 - val_recall: 0.0378
Epoch 4/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4139 - auc: 0.8156 - loss: 1.8752 - precision: 0.7625 - recall: 0.1354 - val_accuracy: 0.3804 - val_auc: 0.7733 - val_loss: 2.0503 - val_precision: 0.6207 - val_recall: 0.0453
Epoch 5/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4658 - auc: 0.8616 - loss: 1.6769 - precision: 0.8261 - recall: 0.2101 - val_accuracy: 0.4005 - val_auc: 0.7823 - val_loss: 2.0135 - val_precision: 0.6559 - val_recall: 0.1537
Epoch 6/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4949 - auc: 0.9019 - loss: 1.4338 - precision: 0.8300 - recall: 0.3012 - val_accuracy: 0.4156 - val_auc: 0.7857 - val_loss: 2.0423 - val_precision: 0.6354 - val_recall: 0.1537
Epoch 7/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 212ms/step - accuracy: 0.5460 - auc: 0.9238 - loss: 1.2751 - precision: 0.8316 - recall: 0.3572 - val_accuracy: 0.4257 - val_auc: 0.7920 - val_loss: 2.0858 - val_precision: 0.6071 - val_recall: 0.2569
Epoch 8/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 211ms/step - accuracy: 0.5891 - auc: 0.9471 - loss: 1.0776 - precision: 0.8539 - recall: 0.4484 - val_accuracy: 0.4181 - val_auc: 0.7821 - val_loss: 2.2802 - val_precision: 0.5674 - val_recall: 0.3073
Epoch 9/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 216ms/step - accuracy: 0.6204 - auc: 0.9566 - loss: 0.9760 - precision: 0.8708 - recall: 0.4972 - val_accuracy: 0.4055 - val_auc: 0.7879 - val_loss: 2.2989 - val_precision: 0.5388 - val_recall: 0.2972
Epoch 10/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 214ms/step - accuracy: 0.6574 - auc: 0.9627 - loss: 0.8983 - precision: 0.8973 - recall: 0.5350 - val_accuracy: 0.4156 - val_auc: 0.7896 - val_loss: 2.2420 - val_precision: 0.5803 - val_recall: 0.2821
Epoch 11/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.6881 - auc: 0.9693 - loss: 0.8198 - precision: 0.8711 - recall: 0.5819 - val_accuracy: 0.4232 - val_auc: 0.7922 - val_loss: 2.4287 - val_precision: 0.5569 - val_recall: 0.3451
Epoch 12/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.7021 - auc: 0.9742 - loss: 0.7545 - precision: 0.8710 - recall: 0.6122 - val_accuracy: 0.4332 - val_auc: 0.7873 - val_loss: 2.4890 - val_precision: 0.5542 - val_recall: 0.3350
Epoch 13/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 214ms/step - accuracy: 0.7285 - auc: 0.9751 - loss: 0.7277 - precision: 0.8831 - recall: 0.6360 - val_accuracy: 0.4181 - val_auc: 0.7833 - val_loss: 2.4983 - val_precision: 0.5391 - val_recall: 0.3300
Epoch 14/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 219ms/step - accuracy: 0.7612 - auc: 0.9800 - loss: 0.6484 - precision: 0.8878 - recall: 0.6555 - val_accuracy: 0.4257 - val_auc: 0.7851 - val_loss: 2.5617 - val_precision: 0.5245 - val_recall: 0.3501
Epoch 15/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.7615 - auc: 0.9785 - loss: 0.6538 - precision: 0.8703 - recall: 0.6647 - val_accuracy: 0.4131 - val_auc: 0.7976 - val_loss: 2.3840 - val_precision: 0.5370 - val_recall: 0.2922
Training time: 162.97 seconds (2.72 minutes)
In [30]:
print("\nValidation Accuracy Model6:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history6.history[m][-1]:.4f}")

Validation Accuracy Model6:
val_accuracy: 0.4131
val_precision: 0.5370
val_recall: 0.2922
val_auc: 0.7976
In [31]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history6.history['accuracy'], label='Train Accuracy')
plt.plot(history6.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history6.history['loss'], label='Train Loss')
plt.plot(history6.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [32]:
y_pred = model2a_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm6 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm6, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 1s 51ms/step
No description has been provided for this image
In [33]:
np.random.seed(160)
tf.random.set_seed(160)


model2b_multi = Sequential()
model2b_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2b_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2b_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2b_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2b_multi.add(Flatten())
model2b_multi.add(Dense(64, activation='relu'))
model2b_multi.add(Dropout(0.4))

model2b_multi.add(Dense(12, activation='softmax'))  

model2b_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

model2b_multi.summary()

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Model: "sequential_5"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ conv2d_10 (Conv2D)              │ (None, 126, 515, 16)   │           160 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ max_pooling2d_10 (MaxPooling2D) │ (None, 63, 257, 16)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv2d_11 (Conv2D)              │ (None, 61, 255, 32)    │         4,640 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ max_pooling2d_11 (MaxPooling2D) │ (None, 30, 127, 32)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ flatten_5 (Flatten)             │ (None, 121920)         │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_10 (Dense)                │ (None, 64)             │     7,802,944 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dropout_5 (Dropout)             │ (None, 64)             │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_11 (Dense)                │ (None, 12)             │           780 │
└─────────────────────────────────┴────────────────────────┴───────────────┘

 Total params: 7,808,524 (29.79 MB)

 Trainable params: 7,808,524 (29.79 MB)

 Non-trainable params: 0 (0.00 B)
In [46]:
# saving bestmodel out of all
model1b_multi.save("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel1.h5")

WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.
In [47]:
model2_multi.save("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel2.h5")

WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.

External Test Data¶

In [48]:
# Loading required libraries
import librosa
import librosa.display
import os
In [61]:
import librosa
import numpy as np
import tensorflow as tf
import os
from collections import Counter


SAMPLE_RATE    = 32000
WINDOW_SIZE    = 10    
STEP_SIZE      = 1.5 
N_MELS         = 128
HOP_LENGTH     = 128
WIN_LENGTH     = 512
EXPECTED_WIDTH = 517

# ─────── Labels & Models ─────────────────────────────────────────────────────
species_labels = species  # your list of 12 species names

model1 = tf.keras.models.load_model("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel1.h5")
model2 = tf.keras.models.load_model("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel2.h5")
models = {"Model 1": model1, "Model 2": model2}

mp3_files = [
    "C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test1.mp3",
    "C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test2.mp3",
    "C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test3.mp3",
]

def sliding_window_preds(audio, sr, model):
    window_len = int(sr * WINDOW_SIZE)
    step_len   = int(sr * STEP_SIZE)
    num_windows = max(1, int((len(audio) - window_len) / step_len) + 1)
    all_preds = []
    
    for i in range(num_windows):
        start, end = i*step_len, i*step_len + window_len
        y_win = audio[start:end].copy()
        
        # spectrogram
        mel = librosa.feature.melspectrogram(
            y=y_win, sr=sr,
            n_mels=N_MELS,
            hop_length=HOP_LENGTH,
            win_length=WIN_LENGTH
        )
        mel_db = librosa.power_to_db(mel, ref=np.max)
        norm  = (mel_db - mel_db.min()) / (mel_db.max() - mel_db.min())
        
        # resize to EXPECTED_WIDTH
        if norm.shape[1] > EXPECTED_WIDTH:
            norm = norm[:, :EXPECTED_WIDTH]
        else:
            pad_width = EXPECTED_WIDTH - norm.shape[1]
            norm = np.pad(norm, ((0,0),(0,pad_width)), mode='constant')
        
        # predict
        inp   = np.expand_dims(norm, axis=(0,-1))  # shape: (1,128,517,1)
        pred  = model.predict(inp, verbose=0)[0]   # vector of 12 probs
        all_preds.append(pred)
    
    return np.stack(all_preds)  # shape: (num_windows, 12)

WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
In [62]:
for model_idx, model in enumerate([model1, model2], start=1):
    print(f"\n============================")
    print(f"Predictions using Model {model_idx}")
    print(f"============================")

    for file in mp3_files:
        print(f"\nFile: {os.path.basename(file)}")

        y, sr = librosa.load(file, sr=SAMPLE_RATE)
        preds = sliding_window_preds(y, sr, model)


        # Average and top predictions
        avg_pred = np.mean(preds, axis=0)
        top3_avg = np.argsort(avg_pred)[::-1][:3]

        print("Top 3 Average Predictions:")
        for i in top3_avg:
            print(f" - {species_labels[i]}: {avg_pred[i]:.2f}")

        # Count most frequent top-1 prediction per window
        top1_per_window = np.argmax(preds, axis=1)
        counts = Counter(top1_per_window)
        print("\nTop species per window:")
        for idx, cnt in counts.most_common():
            print(f" - {species_labels[idx]}: {cnt} windows")

        # Plot the segment with the strongest prediction
        best_idx = np.argmax(np.max(preds, axis=1))
        best_start = int(best_idx * SAMPLE_RATE * STEP_SIZE)
        best_end = best_start + int(SAMPLE_RATE * WINDOW_SIZE)
        y_best = y[best_start:best_end]

        mel = librosa.feature.melspectrogram(y=y_best, sr=sr, n_mels=N_MELS,
                                             hop_length=HOP_LENGTH, win_length=WIN_LENGTH)
        mel_db = librosa.power_to_db(mel, ref=np.max)

        plt.figure(figsize=(10, 4))
        librosa.display.specshow(mel_db, sr=sr, x_axis='time', y_axis='mel',
                                  cmap='gray_r', hop_length=HOP_LENGTH)
        plt.colorbar(format='%+2.0f dB')
        plt.title(f"Model {model_idx} - Strongest Segment - {os.path.basename(file)}")
        plt.tight_layout()
        plt.show()

============================
Predictions using Model 1
============================

File: Homework 3_test_birds_test1.mp3
Top 3 Average Predictions:
 - rewbla: 0.84
 - sonspa: 0.07
 - daejun: 0.06

Top species per window:
 - rewbla: 9 windows
No description has been provided for this image 
File: Homework 3_test_birds_test2.mp3
Top 3 Average Predictions:
 - rewbla: 0.59
 - daejun: 0.16
 - sonspa: 0.14

Top species per window:
 - rewbla: 1 windows
No description has been provided for this image 
File: Homework 3_test_birds_test3.mp3
Top 3 Average Predictions:
 - rewbla: 0.92
 - daejun: 0.04
 - sonspa: 0.03

Top species per window:
 - rewbla: 4 windows
No description has been provided for this image 
============================
Predictions using Model 2
============================

File: Homework 3_test_birds_test1.mp3
Top 3 Average Predictions:
 - houspa: 0.45
 - sonspa: 0.28
 - houfin: 0.27

Top species per window:
 - houspa: 9 windows
No description has been provided for this image 
File: Homework 3_test_birds_test2.mp3
Top 3 Average Predictions:
 - houspa: 0.42
 - sonspa: 0.29
 - houfin: 0.28

Top species per window:
 - houspa: 1 windows
No description has been provided for this image 
File: Homework 3_test_birds_test3.mp3
Top 3 Average Predictions:
 - houspa: 0.48
 - sonspa: 0.27
 - houfin: 0.25

Top species per window:
 - houspa: 4 windows
No description has been provided for this image
In [ ]: